package com.gxljc.bear.crawler.itaogao.gzdaily;

import java.io.Serializable;
import java.lang.reflect.Field;
import java.util.Calendar;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import net.sf.json.JSONArray;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.gxljc.commons.mongo.cli.BaseMongoRepository;
import com.gxljc.bear.crawler.base.BaseMongoTools;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.base.DolphinFetchData;
import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.NewsPageTemplate;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.util.DateUtil;
import com.gxljc.bear.crawler.util.HtmlUtil;
import com.gxljc.bear.crawler.util.JsonUtil;
import com.gxljc.bear.crawler.util.MD5Util;

/**
 * 广州日报更新爬虫数据
 * 
 * @author tanghaitao
 * 
 */
public class GzDailyUpdateCrawler implements Serializable {
	
	private static final long serialVersionUID = 1L;
	public static Logger LOG = Logger.getLogger(GzDailyUpdateCrawler.class);
	private Boolean proxy = false;

	public GzDailyUpdateCrawler(Boolean proxy) {
		this.proxy = proxy;
	}

	public void crawl() throws Exception {
		String dateHtml = getDateHtml();
		if (StringUtils.isBlank(dateHtml)) {
			LOG.error("page is empty");
			return;
		}
		crawlNoSpark(dateHtml);
	}

	public void crawlNoSpark(String dateHtml) {
		try {
			crawlLayout(dateHtml);
		} catch (Exception e) {
			LOG.error("page is empty");
		}
	}

	public void crawlLayout(String html) throws Exception {
		DolphinFetchData fetchData = DolphinFetchData.getInstance(DolphinCrawlerConsts.CrawlerChannelType.bear
				.getName());
		String result = fetchData.getResult(html, proxy, ItaogaoConst.CRAWL_TIMEOUT);
		if (StringUtils.isEmpty(result)) {
			LOG.error("page layout is empty html = " + html);
			return;
		}

		Map<String, Object> resultMap = JsonUtil.parseJSON2Map(result);
		@SuppressWarnings("unchecked")
		List<String> layoutSuffixList = JSONArray.fromObject(resultMap.get("layout"));
		List<String> layoutList = new LinkedList<String>();
		for (String layoutSuffix : layoutSuffixList) {
			String layoutPrefix = html.substring(0, html.lastIndexOf("/") + 1);
			String layoutUrl = layoutPrefix + layoutSuffix;
			layoutList.add(layoutUrl);
		}
		
		for (String url : layoutList) {
			crawlUrl(url);
		}
	}

	// 单条url入口
	public int crawlUrl(String url) throws Exception {
		LOG.info("crawl url = " + url);
		DolphinFetchData fetchData = DolphinFetchData.getInstance(DolphinCrawlerConsts.CrawlerChannelType.bear
				.getName());
		String result = fetchData.getResult(url, proxy, ItaogaoConst.CRAWL_TIMEOUT);
		if (StringUtils.isEmpty(result)) {
			LOG.error("page result is empty url = " + url);
			return DolphinCrawlerConsts.RESULT_NO;
		}
		List<NewsPageTable> tables = extract(result, url);
		MongodbUtil.saveMongodb(tables, ItaogaoConst.MONGODB_TABLE.GZDAILY.getValue());
		return DolphinCrawlerConsts.RESULT_YES;
	}

	private List<NewsPageTable> extract(String result, String url) throws Exception {
		final Gson GSON = new Gson();
		NewsPageTemplate template = GSON.fromJson(result, new TypeToken<NewsPageTemplate>() {
		}.getType());
		if (template == null)
			return null;
		return parseObject(template, url);
	}

	private List<NewsPageTable> parseObject(NewsPageTemplate template, String url) throws Exception {
		List<NewsPageTable> tables = new LinkedList<>();
		List<String> crawlUrls = template.getCrawlUrl();
		if (CollectionUtils.isEmpty(crawlUrls))
			return null;
		int size = crawlUrls.size();
		for (int i = 0; i < size; i++) {
			String crawlUrl = crawlUrls.get(i);
			NewsPageTable table = new NewsPageTable();
			table.setId(genId(crawlUrl));
			table.setCrawlFlag(0);
			table.setCrawlUrl(HtmlUtil.pictureUrlPref(crawlUrl, url));
			tables.add(table);
		}
		return tables;
	}

	// 生成id
	private String genId(String href) {
		return MD5Util.getMd5(href);
	}

	// save data to mongodb
	public void saveMongodb(List<NewsPageTable> tables) throws Exception {
		if (CollectionUtils.isEmpty(tables))
			return;
		for (NewsPageTable table : tables) {
			saveMongodb(table);
		}
	}

	// save data to mongodb
	@SuppressWarnings("rawtypes")
	public void saveMongodb(NewsPageTable table) throws Exception {
		Query query = new Query();
		query.addCriteria(Criteria.where(DolphinCrawlerConsts.MONGODB_DEFAULT_ID).is(table.id));
		Update update = new Update();
		Field[] fields = table.getClass().getDeclaredFields();
		for (int i = 0; i < fields.length; i++) {
			Object value = fields[i].get(table);
			if (value == null)
				continue;
			if (value instanceof List) {
				if (CollectionUtils.isEmpty((List) value))
					continue;
			}
			if (value instanceof String) {
				if (StringUtils.isEmpty((String) value))
					continue;
			}
			update.set(fields[i].getName(), value);
		}
		BaseMongoRepository repo = BaseMongoTools
				.getInstance(DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
		repo.upsert(query, update, ItaogaoConst.MONGODB_TABLE.GZDAILY.getValue());
	}

	public String getDateHtml() throws Exception {
		Date dateSeed = getDateSeed();
		if (dateSeed == null)
			return null;
		String dateFormat = DateUtil.dateFormat(dateSeed, "yyyy-MM/dd");
		String url = String.format(ItaogaoConst.NEWSPAPER_SEED.GZDAILY.getValue(), dateFormat);
		return url;
	}

	/**
	 * 获取当天的日期
	 * 
	 * @return
	 */
	private Date getDateSeed() {
		Calendar now = Calendar.getInstance();
		return now.getTime();
	}

}